package ca.uwindsor.cs.deepweb.estimation;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

public class DictionariesFilesContentIndexer extends FilesContentIndexer {

	protected IndexWriter indexWriter;

	public DictionariesFilesContentIndexer(String sourcepath, String indexdir) {
		super(sourcepath, indexdir);
		// TODO Auto-generated constructor stub
		globalID = 0;
	}

	public DictionariesFilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create, String[] ext) {
		super(sourcepath, indexdir, encoding, verbose, create);
		extensions = ext;
	}

	public void generateIndex() {
		try {
			Analyzer luceneAnalyzer = new StandardAnalyzer();

			// IndexWriter indexWriter;
			indexWriter = new IndexWriter(indexDir, luceneAnalyzer, isCreate);

			long startTime = new Date().getTime();

			generateIndex(dataDir);

			long endTime = new Date().getTime();
			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ dataDir.getPath()
							+ "\nTotal documnents indexed is:" + (globalID - 1));

			System.out
					.println("index is completed! \n Start to optimize the index...");
			indexWriter.optimize();
			System.out.println("optimization is completed!");
			indexWriter.close();

			// } catch (CorruptIndexException e) {
			// // TODO Auto-generated catch block
			// e.printStackTrace();
			// System.exit(1);
			// } catch (LockObtainFailedException e) {
			// // TODO Auto-generated catch block
			// e.printStackTrace();
			// System.exit(1);
			// } catch (IOException e) {
			// // TODO Auto-generated catch block
			// e.printStackTrace();
			// System.exit(1);
			// }
		} catch (Exception e) {
			e.printStackTrace();
			try {
				indexWriter.close();
			} catch (Exception e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
				System.out.println("Close index failed!");
				System.out.println("There are: " + globalID + "files indexed!");
				System.exit(-1);
			}
			System.out.println("The Index has been closed!");
			System.out.println("There are: " + globalID + "files indexed!");
			System.exit(-1);
		}
	}

	protected void generateIndex(File directory) {
		try {
			File[] dataFiles = directory.listFiles();
			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				File file = dataFiles[i];
				if (file.isFile()) {
					if (isValidExtension(file.getName())) {
						if (isVerbose) {
							System.out.println("Indexing file "
									+ file.getCanonicalPath());
						}
						Document document = new Document();
						InputStreamReader txtReader = new InputStreamReader(
								new FileInputStream(file), encoding);

						// make the ID from the path shorter - get the file name
						String filepath = file.getCanonicalPath();
						int offset1 = filepath
								.lastIndexOf(java.io.File.separator);
						int offset2 = filepath.lastIndexOf(".");
						String word = new String(filepath.substring(
								offset1 + 1, offset2));
						// String word = String.valueOf(globalID);

						Field FieldPath = new Field(
								FilesContentIndexer.FIELD_ID, word,
								Field.Store.YES, Field.Index.UN_TOKENIZED);
						Field FieldTitle = new Field(
								FilesContentIndexer.FIELD_TITLE, word,
								Field.Store.YES, Field.Index.UN_TOKENIZED);
						Field FieldSize = new Field(
								FilesContentIndexer.FIELD_SIZE, Integer
										.toString(new Long(file.length()).intValue()), Field.Store.YES,
								Field.Index.UN_TOKENIZED);
						Field FieldBody = new Field(
								FilesContentIndexer.FIELD_CONTENT, txtReader,
								Field.TermVector.YES);

						document.add(FieldPath);
						document.add(FieldTitle);
						document.add(FieldSize);
						document.add(FieldBody);
						indexWriter.addDocument(document);
						txtReader.close();
						globalID++;
					}
				} else if (file.isDirectory()) {
					generateIndex(file);
				}
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	public static void main(String[] args) {
		// String[] ext = { "html" };
		// SogouXMLFilesIndexer indexer = new SogouXMLFilesIndexer(
		// "G:/gov2-corpus-extracted", "F:/indexes/GOV2", "UTF-8", true,
		// true, ext);
		// indexer.generateIndex();

		 args = new String[6];
		 args[0] = "--SourceDir=D:\\Research\\estimation\\datasource\\NG_regrouped";
		 args[1] = "--IndexDir=F:\\NG_regrouped";
		 args[2] = "--encoding=latin1";
		 args[3] = "-verbose";
		 args[4]= "-create";
		 args[5]= "--extension=txt";

		String SourceDir = new String();
		String IndexDir = new String();
		String encoding = new String("UTF-8");
		boolean isVerbose = false;
		boolean isCreate = true;

		String ext[] = new String[] { "txt" };

		if (args.length < 2) {
			System.out
					.println("usage: java -jar DictionariesFilesContentIndexer --SourceDir=sourcefiledirectory --IndexDir=indexfiledirectory --extension=txt,html,... [--encoding=encoding](default=UTF-8) [-verbose] [-create] !");
			System.exit(0);
		} else {
			for (int i = 0; i < args.length; i++) {
				if (args[i].startsWith("--SourceDir=")) {
					SourceDir = args[i].substring(12);
				} else if (args[i].startsWith("--IndexDir=")) {
					IndexDir = args[i].substring(11);
				} else if (args[i].startsWith("--encoding=")) {
					encoding = args[i].substring(11);
				} else if (args[i].equals("-verbose")) {
					isVerbose = true;
				} else if (args[i].equals("-create")) {
					isCreate = true;
				} else if (args[i].startsWith("--extension=")) {
					String tempexts = args[i].substring(12);
					ext = tempexts.split(",");
				} else {
					System.out
							.println("usage: java -jar DictionariesFilesContentIndexer SourceDir=sourcefiledirectory IndexDir=indexfiledirectory [encoding=encoding](default=UTF-8) [-verbose]!");
					System.exit(0);
				}
			}
		}

		if (SourceDir.equals("") || IndexDir.equals("")) {
			System.out
					.println("usage: java -jar DictionariesFilesContentIndexer SourceDir=sourcefiledirectory IndexDir=indexfiledirectory [encoding=encoding](default=UTF-8) [-verbose]!");
			System.exit(0);
		} else {
			DictionariesFilesContentIndexer indexer;
			// if (isRecursive) {
			// indexer = new SogouXMLFilesIndexer(SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// } else {
			// indexer = new SogouXMLFilesIndexer (SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// }
			indexer = new DictionariesFilesContentIndexer(SourceDir, IndexDir,
					encoding, isVerbose, isCreate, ext);
			indexer.generateIndex();
		}
	}
}
